notebook.community

Edit and run

Task Overview

Data: Each entry is a fixed-size sequence of values, generated from 3 different distribution

Objective: Predict the T next values given the previous S values of the seqeunce





In [1]:



    


import pandas as pd
import numpy as np
import shutil
from datetime import datetime
import multiprocessing

import tensorflow as tf
from tensorflow import data
import tensorflow.contrib.rnn as rnn

import matplotlib.pyplot as plt

print(tf.__version__)



    


















    






/Users/khalidsalama/anaconda/lib/python3.6/importlib/_bootstrap.py:205: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6
  return f(*args, **kwds)










    






1.4.0

















In [2]:



    


MODEL_NAME = 'seq-model-02'

SEQUENCE_LENGTH = 20
INPUT_SEQUENCE_LENGTH = 16
OUTPUT_SEQUENCE_LENGTH = SEQUENCE_LENGTH - INPUT_SEQUENCE_LENGTH

TRAIN_DATA_SIZE = 1200 # sequences (entries)
TEST_DATA_SIZE = 300

TRAIN_DATA_FILE = 'data/seq02.train.csv'
TEST_DATA_FILE = 'data/seq02.test.csv'

RESUME_TRAINING = False
MULTI_THREADING = True

SKIP_DATA_GENERATION = True

Generate Sequence Data





In [3]:



    


NOISE_RANGE = 0.01
TREND = 10.
OSCILIATION = 5.
np.random.seed = 19831006

def create_sequence1(start_value):

    x =  np.array(range(start_value, start_value+SEQUENCE_LENGTH))
    noise = np.random.normal(0, NOISE_RANGE, SEQUENCE_LENGTH)
    y = np.sin(np.pi * x / OSCILIATION) + (x / TREND + NOISE_RANGE)
    return y


def create_sequence2(start_value):

    x =  np.array(range(start_value, start_value+SEQUENCE_LENGTH))
    noise = np.random.normal(0, NOISE_RANGE+0.5, SEQUENCE_LENGTH)
    y = -x + noise
    return y


def create_sequence3(start_value):

    x =  np.array(range(start_value, start_value+SEQUENCE_LENGTH))
    y = []
    
    for x_i in x:
        y_i = 0
        if x_i % 2 == 0:
            y_i = x_i * 2
        else:
            y_i =  - x_i * 2
        y += [y_i]
        
    return y

plt.close('all')
plt.figure(figsize=(20, 15))

plt.subplot(3,1,1)
plt.title("Sequences Pattern 1")

for i in range(-5, 5):
    
    x = range(SEQUENCE_LENGTH)
    y = create_sequence1(i)
    plt.plot(x,y)
    
    
plt.subplot(3,1,2)
plt.title("Sequences Pattern 2")

for i in range(-5, 5):
    
    x = range(SEQUENCE_LENGTH)
    y = create_sequence2(i)
    plt.plot(x,y)
    

plt.subplot(3,1,3)
plt.title("Sequences Pattern 3")

for i in range(-5, 5):
    
    x = range(SEQUENCE_LENGTH)
    y = create_sequence3(i)
    plt.plot(x,y)


plt.xlabel("Element Index")
plt.ylabel("Value")
plt.show()



    


















    
























In [4]:



    


patterns = 3
start = int(-1*TRAIN_DATA_SIZE/(2*patterns))
end = int(TRAIN_DATA_SIZE/(2*patterns))

def to_csv():
    
    with open(TRAIN_DATA_FILE, 'w') as train_file_writer:
        with open(TEST_DATA_FILE, 'w') as test_file_writer:
            
            for line_index in range(start, end):
                
                sequence1 = create_sequence1(line_index)
                csv_line1 = ",".join(map(str, sequence1))
                
                sequence2 = create_sequence2(line_index)
                csv_line2 = ",".join(map(str, sequence2))
                
                sequence3 = create_sequence3(line_index)
                csv_line3 = ",".join(map(str, sequence3))
                
                if line_index  % 6 == 0:
                    test_file_writer.write(csv_line1 + '\n')
                    test_file_writer.write(csv_line2 + '\n')
                    test_file_writer.write(csv_line3 + '\n')
                else:
                    train_file_writer.write(csv_line1 + '\n')
                    train_file_writer.write(csv_line2 + '\n')
                    train_file_writer.write(csv_line3 + '\n')
                    
if not SKIP_DATA_GENERATION:               
    to_csv()



    











In [5]:



    


df1 = pd.read_csv(TRAIN_DATA_FILE,header=None)
df2 = pd.read_csv(TEST_DATA_FILE,header=None)
print(len(df1))
print(len(df2))



    


















    






999
201

















In [6]:



    


plt.close('all')
plt.figure(figsize=(20, 5))

plt.subplot(1,3,1)
plt.title("Sequences Pattern 1")
x = range(SEQUENCE_LENGTH)
y = df1.iloc[0,:]
plt.plot(x,y)

plt.subplot(1,3,2)
plt.title("Sequences Pattern 2")
x = range(SEQUENCE_LENGTH)
y = df1.iloc[1,:]
plt.plot(x,y)

plt.subplot(1,3,3)
plt.title("Sequences Pattern 3")
x = range(SEQUENCE_LENGTH)
y = df1.iloc[2,:]
plt.plot(x,y)
    
plt.xlabel("Element Index")
plt.ylabel("Value")
plt.show()

1. Define Dataset Metadata





In [7]:



    


DEFAULTS = [[0.0] for i in range(0, SEQUENCE_LENGTH)]
VALUES_FEATURE_NAME = 'values'

2. Define Data Input Function





In [8]:



    


def parse_csv_row(csv_row):
    
    # sequence is a list of tensors
    sequence = tf.decode_csv(tf.expand_dims(csv_row, -1), record_defaults=DEFAULTS)  
    
    input_sequence = sequence[ :INPUT_SEQUENCE_LENGTH]  # input elements in the sequence
    output_sequence = sequence[INPUT_SEQUENCE_LENGTH: ] # output elements in the sequence
    
    input_sequence_tensor = tf.concat(input_sequence, axis=1)
    output_sequence_tensor = tf.concat(output_sequence, axis=1)
    
    return {VALUES_FEATURE_NAME: input_sequence_tensor}, output_sequence_tensor



    











In [9]:



    


def csv_input_fn(files_name_pattern, mode=tf.estimator.ModeKeys.EVAL, 
                 skip_header_lines=0, 
                 num_epochs=1, 
                 batch_size=20):
    
    shuffle = True if mode == tf.estimator.ModeKeys.TRAIN else False
    
    num_threads = multiprocessing.cpu_count() if MULTI_THREADING else 1
    
    print("")
    print("* data input_fn:")
    print("================")
    print("Input file(s): {}".format(files_name_pattern))
    print("Batch size: {}".format(batch_size))
    print("Epoch Count: {}".format(num_epochs))
    print("Mode: {}".format(mode))
    print("Shuffle: {}".format(shuffle))
    print("================")
    print("")
    
    file_names = tf.matching_files(files_name_pattern)

    dataset = data.TextLineDataset(filenames=file_names)
    dataset = dataset.skip(skip_header_lines)
    
    if shuffle:
        dataset = dataset.shuffle(buffer_size=2 * batch_size + 1)

    dataset = dataset.batch(batch_size)
    dataset = dataset.map(lambda csv_row: parse_csv_row(csv_row), 
                          num_parallel_calls=num_threads)
    
    #dataset = dataset.batch(batch_size) #??? very long time
    dataset = dataset.repeat(num_epochs)
    iterator = dataset.make_one_shot_iterator()
    
    features, target = iterator.get_next()
    return features, target



    











In [10]:



    


features, target = csv_input_fn(files_name_pattern="")
print("Features in CSV: {}".format(features))
print("Target in CSV: {}".format(target))



    


















    






* data input_fn:
================
Input file(s): 
Batch size: 20
Epoch Count: 1
Mode: eval
Shuffle: False
================

Features in CSV: {'values': <tf.Tensor 'IteratorGetNext:0' shape=(?, 16) dtype=float32>}
Target in CSV: Tensor("IteratorGetNext:1", shape=(?, 4), dtype=float32)

3. Define RNN Model Function





In [11]:



    


def rnn_model_fn(features, labels, mode, params):
    
    # 0. Reformat input shape to become a sequence
    inputs = tf.split(features[VALUES_FEATURE_NAME], INPUT_SEQUENCE_LENGTH, 1)
    
#     # 1. configure the RNN
#     lstm_cell = rnn.BasicLSTMCell(
#         num_units=params.hidden_units, 
#         forget_bias=params.forget_bias,
#         activation=tf.nn.tanh
#     )
    
    ## 1. configure the RNN
    rnn_layers = [tf.nn.rnn_cell.LSTMCell(
        num_units=size, 
        forget_bias=params.forget_bias,
        activation=tf.nn.tanh) for size in hparams.hidden_units]

    # create a RNN cell composed sequentially of a number of RNNCells
    multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers)

    outputs, _ = rnn.static_rnn(cell=multi_rnn_cell, 
                                inputs=inputs, 
                                dtype=tf.float32)
    
    # slice to keep only the last cell of the RNN
    outputs = outputs[-1]
    
    predictions = tf.layers.dense(inputs=outputs,
                                  units=OUTPUT_SEQUENCE_LENGTH,
                                  activation=None)
    
    predict_output = {'values': predictions}
    
    if mode == tf.estimator.ModeKeys.PREDICT:

        export_outputs = {
                'predictions': tf.estimator.export.PredictOutput(predict_output)
            }

        return tf.estimator.EstimatorSpec(
                mode=mode,
                predictions=predict_output,
                export_outputs=export_outputs)


    # Calculate loss using mean squared error
    loss = tf.losses.mean_squared_error(labels, predictions)

    # Create Optimiser
    optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate)

    # Create training operation
    train_op = optimizer.minimize(
        loss=loss, global_step=tf.train.get_global_step())

    # Calculate root mean squared error as additional eval metric
    eval_metric_ops = {
        "rmse": tf.metrics.root_mean_squared_error(labels, predictions),
        "mae": tf.metrics.mean_absolute_error(labels, predictions)
    }

    # Provide an estimator spec for `ModeKeys.EVAL` and `ModeKeys.TRAIN` modes.
    estimator_spec = tf.estimator.EstimatorSpec(mode=mode,
                                                loss=loss,
                                                train_op=train_op,
                                                eval_metric_ops=eval_metric_ops)

    return estimator_spec


def create_estimator(run_config, hparams):
    estimator = tf.estimator.Estimator(model_fn=rnn_model_fn, 
                                  params=hparams, 
                                  config=run_config)
    
    print("")
    print("Estimator Type: {}".format(type(estimator)))
    print("")

    return estimator

5. Run Experiment

a. Set HParam and RunConfig





In [12]:



    


NUM_EPOCHS = 5000
BATCH_SIZE = 200
EVAL_AFTER_SEC = 15
TOTAL_STEPS = (TRAIN_DATA_SIZE/BATCH_SIZE)*NUM_EPOCHS

hparams  = tf.contrib.training.HParams(
    num_epochs = NUM_EPOCHS,
    batch_size = BATCH_SIZE,
    hidden_units=[20, 10],
    forget_bias=1.0,
    learning_rate=0.01,
    keep_prob = 0.8,
    max_steps = TOTAL_STEPS
)

model_dir = 'trained_models/{}'.format(MODEL_NAME)

run_config = tf.estimator.RunConfig(
    tf_random_seed=19830610,
    model_dir=model_dir
)

print(hparams)
print("Model Directory:", run_config.model_dir)
print("")
print("Dataset Size:", TRAIN_DATA_SIZE)
print("Batch Size:", BATCH_SIZE)
print("Steps per Epoch:",TRAIN_DATA_SIZE/BATCH_SIZE)
print("Total Steps:", TOTAL_STEPS)
print("That is 1 evaluation step after each",EVAL_AFTER_SEC," training seconds")



    


















    






[('batch_size', 200), ('forget_bias', 1.0), ('hidden_units', [20, 10]), ('keep_prob', 0.8), ('learning_rate', 0.01), ('max_steps', 30000.0), ('num_epochs', 5000)]
Model Directory: trained_models/seq-model-02

Dataset Size: 1200
Batch Size: 200
Steps per Epoch: 6.0
Total Steps: 30000.0
That is 1 evaluation step after each 15  training seconds

b. Define Serving Function





In [13]:



    


def csv_serving_input_fn():
    
    SERVING_DEFAULTS =  [[0.0] for i in range(0, INPUT_SEQUENCE_LENGTH)]

    rows_string_tensor = tf.placeholder(dtype=tf.string,
                                         shape=[None],
                                         name='csv_rows')
    
    receiver_tensor = {'csv_rows': rows_string_tensor}

    row_columns = tf.expand_dims(rows_string_tensor, -1)
    columns = tf.decode_csv(row_columns, record_defaults=SERVING_DEFAULTS)
    input_sequence_tensor = tf.concat(columns, axis=1)
    features = {VALUES_FEATURE_NAME: input_sequence_tensor}

    return tf.estimator.export.ServingInputReceiver(
        features, receiver_tensor)

c. Define TrainSpec and EvaluSpec





In [14]:



    


train_spec = tf.estimator.TrainSpec(
    input_fn = lambda: csv_input_fn(
        TRAIN_DATA_FILE,
        mode = tf.estimator.ModeKeys.TRAIN,
        num_epochs=hparams.num_epochs,
        batch_size=hparams.batch_size
    ),
    max_steps=hparams.max_steps,
    hooks=None
)

eval_spec = tf.estimator.EvalSpec(
    input_fn = lambda: csv_input_fn(
        TRAIN_DATA_FILE,
        mode=tf.estimator.ModeKeys.EVAL,
        num_epochs=1,
        batch_size=hparams.batch_size,
            
    ),
    exporters=[tf.estimator.LatestExporter(
        name="forecast",  # the name of the folder in which the model will be exported to under export
        serving_input_receiver_fn=csv_serving_input_fn,
        exports_to_keep=1,
        as_text=True)],
    #throttle_secs = EVAL_AFTER_SEC,
    steps=None
)

d. Run the experiment via tf.estimator.train_and_evaluate()





In [15]:



    


if not RESUME_TRAINING:
    print("Removing previous artifacts...")
    shutil.rmtree(model_dir, ignore_errors=True)
else:
    print("Resuming training...") 

    
tf.logging.set_verbosity(tf.logging.INFO)

time_start = datetime.utcnow() 
print("Experiment started at {}".format(time_start.strftime("%H:%M:%S")))
print(".......................................") 

estimator = create_estimator(run_config, hparams)

tf.estimator.train_and_evaluate(
    estimator=estimator,
    train_spec=train_spec, 
    eval_spec=eval_spec
)

time_end = datetime.utcnow() 
print(".......................................")
print("Experiment finished at {}".format(time_end.strftime("%H:%M:%S")))
print("")
time_elapsed = time_end - time_start
print("Experiment elapsed time: {} seconds".format(time_elapsed.total_seconds()))



    


















    






Removing previous artifacts...
Experiment started at 18:44:48
.......................................
INFO:tensorflow:Using config: {'_model_dir': 'trained_models/seq-model-02', '_tf_random_seed': 19830610, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x126aa35c0>, '_task_type': 'worker', '_task_id': 0, '_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}

Estimator Type: <class 'tensorflow.python.estimator.estimator.Estimator'>

INFO:tensorflow:Running training and evaluation locally (non-distributed).
INFO:tensorflow:Start train and evaluate loop. The evaluate will happen after 600 secs (eval_spec.throttle_secs) or training is finished.

* data input_fn:
================
Input file(s): data/seq02.train.csv
Batch size: 200
Epoch Count: 5000
Mode: train
Shuffle: True
================

INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Saving checkpoints for 1 into trained_models/seq-model-02/model.ckpt.
INFO:tensorflow:loss = 15696.1, step = 1
INFO:tensorflow:global_step/sec: 57.792
INFO:tensorflow:loss = 19331.8, step = 101 (1.732 sec)
INFO:tensorflow:global_step/sec: 98.8345
INFO:tensorflow:loss = 16248.9, step = 201 (1.012 sec)
INFO:tensorflow:global_step/sec: 99.805
INFO:tensorflow:loss = 12321.5, step = 301 (1.002 sec)
INFO:tensorflow:global_step/sec: 99.6004
INFO:tensorflow:loss = 14189.1, step = 401 (1.004 sec)
INFO:tensorflow:global_step/sec: 100.735
INFO:tensorflow:loss = 13920.8, step = 501 (0.993 sec)
INFO:tensorflow:global_step/sec: 100.447
INFO:tensorflow:loss = 13108.1, step = 601 (0.996 sec)
INFO:tensorflow:global_step/sec: 91.3887
INFO:tensorflow:loss = 11071.6, step = 701 (1.094 sec)
INFO:tensorflow:global_step/sec: 94.5503
INFO:tensorflow:loss = 10469.2, step = 801 (1.057 sec)
INFO:tensorflow:global_step/sec: 100.017
INFO:tensorflow:loss = 10823.8, step = 901 (1.000 sec)
INFO:tensorflow:global_step/sec: 100.209
INFO:tensorflow:loss = 10458.1, step = 1001 (0.998 sec)
INFO:tensorflow:global_step/sec: 100.521
INFO:tensorflow:loss = 8734.09, step = 1101 (0.995 sec)
INFO:tensorflow:global_step/sec: 100.771
INFO:tensorflow:loss = 8031.83, step = 1201 (0.992 sec)
INFO:tensorflow:global_step/sec: 100.33
INFO:tensorflow:loss = 9064.62, step = 1301 (0.997 sec)
INFO:tensorflow:global_step/sec: 100.811
INFO:tensorflow:loss = 7081.39, step = 1401 (0.992 sec)
INFO:tensorflow:global_step/sec: 100.029
INFO:tensorflow:loss = 8562.81, step = 1501 (1.000 sec)
INFO:tensorflow:global_step/sec: 100.219
INFO:tensorflow:loss = 6354.14, step = 1601 (0.998 sec)
INFO:tensorflow:global_step/sec: 100.674
INFO:tensorflow:loss = 5338.34, step = 1701 (0.993 sec)
INFO:tensorflow:global_step/sec: 99.2634
INFO:tensorflow:loss = 5754.87, step = 1801 (1.007 sec)
INFO:tensorflow:global_step/sec: 100.602
INFO:tensorflow:loss = 6434.48, step = 1901 (0.994 sec)
INFO:tensorflow:global_step/sec: 99.2796
INFO:tensorflow:loss = 5577.54, step = 2001 (1.007 sec)
INFO:tensorflow:global_step/sec: 100.158
INFO:tensorflow:loss = 4648.08, step = 2101 (0.998 sec)
INFO:tensorflow:global_step/sec: 100.286
INFO:tensorflow:loss = 4245.31, step = 2201 (0.997 sec)
INFO:tensorflow:global_step/sec: 98.2467
INFO:tensorflow:loss = 4253.37, step = 2301 (1.018 sec)
INFO:tensorflow:global_step/sec: 97.207
INFO:tensorflow:loss = 4068.83, step = 2401 (1.029 sec)
INFO:tensorflow:global_step/sec: 100.082
INFO:tensorflow:loss = 3361.38, step = 2501 (0.999 sec)
INFO:tensorflow:global_step/sec: 98.1058
INFO:tensorflow:loss = 4393.81, step = 2601 (1.019 sec)
INFO:tensorflow:global_step/sec: 100.739
INFO:tensorflow:loss = 3542.13, step = 2701 (0.993 sec)
INFO:tensorflow:global_step/sec: 99.9715
INFO:tensorflow:loss = 3540.82, step = 2801 (1.000 sec)
INFO:tensorflow:global_step/sec: 100.524
INFO:tensorflow:loss = 3272.48, step = 2901 (0.995 sec)
INFO:tensorflow:global_step/sec: 91.333
INFO:tensorflow:loss = 2931.14, step = 3001 (1.095 sec)
INFO:tensorflow:global_step/sec: 81.1939
INFO:tensorflow:loss = 3314.65, step = 3101 (1.232 sec)
INFO:tensorflow:global_step/sec: 96.3243
INFO:tensorflow:loss = 2639.88, step = 3201 (1.038 sec)
INFO:tensorflow:global_step/sec: 100.356
INFO:tensorflow:loss = 2091.61, step = 3301 (0.996 sec)
INFO:tensorflow:global_step/sec: 87.7768
INFO:tensorflow:loss = 2550.01, step = 3401 (1.139 sec)
INFO:tensorflow:global_step/sec: 103.742
INFO:tensorflow:loss = 2519.9, step = 3501 (0.964 sec)
INFO:tensorflow:global_step/sec: 102.605
INFO:tensorflow:loss = 2434.88, step = 3601 (0.975 sec)
INFO:tensorflow:global_step/sec: 106.848
INFO:tensorflow:loss = 2244.1, step = 3701 (0.936 sec)
INFO:tensorflow:global_step/sec: 106.922
INFO:tensorflow:loss = 1887.14, step = 3801 (0.935 sec)
INFO:tensorflow:global_step/sec: 106.562
INFO:tensorflow:loss = 1846.06, step = 3901 (0.939 sec)
INFO:tensorflow:global_step/sec: 105.571
INFO:tensorflow:loss = 1346.92, step = 4001 (0.947 sec)
INFO:tensorflow:global_step/sec: 106.694
INFO:tensorflow:loss = 1947.21, step = 4101 (0.937 sec)
INFO:tensorflow:global_step/sec: 105.532
INFO:tensorflow:loss = 1500.63, step = 4201 (0.948 sec)
INFO:tensorflow:global_step/sec: 96.2562
INFO:tensorflow:loss = 1517.87, step = 4301 (1.039 sec)
INFO:tensorflow:global_step/sec: 106.961
INFO:tensorflow:loss = 1479.26, step = 4401 (0.935 sec)
INFO:tensorflow:global_step/sec: 106.802
INFO:tensorflow:loss = 1146.53, step = 4501 (0.936 sec)
INFO:tensorflow:global_step/sec: 106.794
INFO:tensorflow:loss = 906.104, step = 4601 (0.936 sec)
INFO:tensorflow:global_step/sec: 107.47
INFO:tensorflow:loss = 1073.1, step = 4701 (0.930 sec)
INFO:tensorflow:global_step/sec: 106.774
INFO:tensorflow:loss = 943.67, step = 4801 (0.937 sec)
INFO:tensorflow:global_step/sec: 107.966
INFO:tensorflow:loss = 901.328, step = 4901 (0.926 sec)
INFO:tensorflow:global_step/sec: 107.243
INFO:tensorflow:loss = 766.044, step = 5001 (0.933 sec)
INFO:tensorflow:global_step/sec: 106.943
INFO:tensorflow:loss = 743.582, step = 5101 (0.935 sec)
INFO:tensorflow:global_step/sec: 108.076
INFO:tensorflow:loss = 908.824, step = 5201 (0.925 sec)
INFO:tensorflow:global_step/sec: 109.349
INFO:tensorflow:loss = 834.799, step = 5301 (0.914 sec)
INFO:tensorflow:global_step/sec: 107.519
INFO:tensorflow:loss = 593.867, step = 5401 (0.930 sec)
INFO:tensorflow:global_step/sec: 104.971
INFO:tensorflow:loss = 568.866, step = 5501 (0.953 sec)
INFO:tensorflow:global_step/sec: 105.23
INFO:tensorflow:loss = 685.489, step = 5601 (0.950 sec)
INFO:tensorflow:global_step/sec: 106.954
INFO:tensorflow:loss = 595.544, step = 5701 (0.935 sec)
INFO:tensorflow:global_step/sec: 105.351
INFO:tensorflow:loss = 507.285, step = 5801 (0.949 sec)
INFO:tensorflow:global_step/sec: 105.428
INFO:tensorflow:loss = 502.705, step = 5901 (0.949 sec)
INFO:tensorflow:global_step/sec: 107.289
INFO:tensorflow:loss = 449.941, step = 6001 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.657
INFO:tensorflow:loss = 284.967, step = 6101 (0.929 sec)
INFO:tensorflow:global_step/sec: 106.956
INFO:tensorflow:loss = 411.203, step = 6201 (0.935 sec)
INFO:tensorflow:global_step/sec: 106.589
INFO:tensorflow:loss = 435.899, step = 6301 (0.938 sec)
INFO:tensorflow:global_step/sec: 107.198
INFO:tensorflow:loss = 378.057, step = 6401 (0.933 sec)
INFO:tensorflow:global_step/sec: 108.036
INFO:tensorflow:loss = 384.802, step = 6501 (0.926 sec)
INFO:tensorflow:global_step/sec: 107.265
INFO:tensorflow:loss = 241.779, step = 6601 (0.932 sec)
INFO:tensorflow:global_step/sec: 105.521
INFO:tensorflow:loss = 254.94, step = 6701 (0.948 sec)
INFO:tensorflow:global_step/sec: 105.879
INFO:tensorflow:loss = 237.654, step = 6801 (0.944 sec)
INFO:tensorflow:global_step/sec: 107.177
INFO:tensorflow:loss = 214.038, step = 6901 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.714
INFO:tensorflow:loss = 223.9, step = 7001 (0.929 sec)
INFO:tensorflow:global_step/sec: 108.448
INFO:tensorflow:loss = 194.831, step = 7101 (0.922 sec)
INFO:tensorflow:global_step/sec: 108.001
INFO:tensorflow:loss = 204.202, step = 7201 (0.926 sec)
INFO:tensorflow:global_step/sec: 107.648
INFO:tensorflow:loss = 242.524, step = 7301 (0.929 sec)
INFO:tensorflow:global_step/sec: 107.81
INFO:tensorflow:loss = 726.775, step = 7401 (0.928 sec)
INFO:tensorflow:global_step/sec: 106.776
INFO:tensorflow:loss = 226.5, step = 7501 (0.936 sec)
INFO:tensorflow:global_step/sec: 107.125
INFO:tensorflow:loss = 167.556, step = 7601 (0.934 sec)
INFO:tensorflow:global_step/sec: 107.218
INFO:tensorflow:loss = 145.338, step = 7701 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.459
INFO:tensorflow:loss = 161.965, step = 7801 (0.931 sec)
INFO:tensorflow:global_step/sec: 107.506
INFO:tensorflow:loss = 110.012, step = 7901 (0.930 sec)
INFO:tensorflow:global_step/sec: 107.332
INFO:tensorflow:loss = 125.73, step = 8001 (0.932 sec)
INFO:tensorflow:global_step/sec: 106.566
INFO:tensorflow:loss = 117.435, step = 8101 (0.939 sec)
INFO:tensorflow:global_step/sec: 107.279
INFO:tensorflow:loss = 81.0131, step = 8201 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.226
INFO:tensorflow:loss = 76.1837, step = 8301 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.202
INFO:tensorflow:loss = 102.379, step = 8401 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.197
INFO:tensorflow:loss = 67.4578, step = 8501 (0.933 sec)
INFO:tensorflow:global_step/sec: 108.272
INFO:tensorflow:loss = 92.3618, step = 8601 (0.924 sec)
INFO:tensorflow:global_step/sec: 105.646
INFO:tensorflow:loss = 57.9386, step = 8701 (0.947 sec)
INFO:tensorflow:global_step/sec: 106.829
INFO:tensorflow:loss = 65.0466, step = 8801 (0.936 sec)
INFO:tensorflow:global_step/sec: 109.056
INFO:tensorflow:loss = 94.4898, step = 8901 (0.917 sec)
INFO:tensorflow:global_step/sec: 109.023
INFO:tensorflow:loss = 58.1859, step = 9001 (0.917 sec)
INFO:tensorflow:global_step/sec: 106.885
INFO:tensorflow:loss = 47.1054, step = 9101 (0.936 sec)
INFO:tensorflow:global_step/sec: 106.633
INFO:tensorflow:loss = 84.8044, step = 9201 (0.938 sec)
INFO:tensorflow:global_step/sec: 106.915
INFO:tensorflow:loss = 49.3459, step = 9301 (0.935 sec)
INFO:tensorflow:global_step/sec: 107.549
INFO:tensorflow:loss = 68.7487, step = 9401 (0.930 sec)
INFO:tensorflow:global_step/sec: 108.557
INFO:tensorflow:loss = 77.0887, step = 9501 (0.921 sec)
INFO:tensorflow:global_step/sec: 107.646
INFO:tensorflow:loss = 81.0733, step = 9601 (0.929 sec)
INFO:tensorflow:global_step/sec: 106.782
INFO:tensorflow:loss = 58.7292, step = 9701 (0.937 sec)
INFO:tensorflow:global_step/sec: 107.198
INFO:tensorflow:loss = 38.8841, step = 9801 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.695
INFO:tensorflow:loss = 62.2282, step = 9901 (0.929 sec)
INFO:tensorflow:global_step/sec: 107.387
INFO:tensorflow:loss = 32.8771, step = 10001 (0.931 sec)
INFO:tensorflow:global_step/sec: 106.956
INFO:tensorflow:loss = 22.251, step = 10101 (0.935 sec)
INFO:tensorflow:global_step/sec: 108.979
INFO:tensorflow:loss = 41.2366, step = 10201 (0.918 sec)
INFO:tensorflow:global_step/sec: 108.66
INFO:tensorflow:loss = 30.9339, step = 10301 (0.920 sec)
INFO:tensorflow:global_step/sec: 107.463
INFO:tensorflow:loss = 52.4443, step = 10401 (0.931 sec)
INFO:tensorflow:global_step/sec: 104.888
INFO:tensorflow:loss = 22.72, step = 10501 (0.953 sec)
INFO:tensorflow:global_step/sec: 108.646
INFO:tensorflow:loss = 39.1618, step = 10601 (0.920 sec)
INFO:tensorflow:global_step/sec: 107.16
INFO:tensorflow:loss = 41.4933, step = 10701 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.6
INFO:tensorflow:loss = 28.8167, step = 10801 (0.929 sec)
INFO:tensorflow:global_step/sec: 106.773
INFO:tensorflow:loss = 36.0264, step = 10901 (0.937 sec)
INFO:tensorflow:global_step/sec: 108.712
INFO:tensorflow:loss = 13.6514, step = 11001 (0.920 sec)
INFO:tensorflow:global_step/sec: 107.317
INFO:tensorflow:loss = 41.6631, step = 11101 (0.932 sec)
INFO:tensorflow:global_step/sec: 106.463
INFO:tensorflow:loss = 33.6448, step = 11201 (0.939 sec)
INFO:tensorflow:global_step/sec: 107.234
INFO:tensorflow:loss = 26.4411, step = 11301 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.583
INFO:tensorflow:loss = 54.4711, step = 11401 (0.930 sec)
INFO:tensorflow:global_step/sec: 107.311
INFO:tensorflow:loss = 92.543, step = 11501 (0.932 sec)
INFO:tensorflow:global_step/sec: 109.093
INFO:tensorflow:loss = 240.377, step = 11601 (0.917 sec)
INFO:tensorflow:global_step/sec: 107.921
INFO:tensorflow:loss = 266.065, step = 11701 (0.927 sec)
INFO:tensorflow:global_step/sec: 106.927
INFO:tensorflow:loss = 384.216, step = 11801 (0.935 sec)
INFO:tensorflow:global_step/sec: 105.633
INFO:tensorflow:loss = 65.2427, step = 11901 (0.947 sec)
INFO:tensorflow:global_step/sec: 106.834
INFO:tensorflow:loss = 48.5648, step = 12001 (0.936 sec)
INFO:tensorflow:global_step/sec: 107.265
INFO:tensorflow:loss = 49.424, step = 12101 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.034
INFO:tensorflow:loss = 29.181, step = 12201 (0.934 sec)
INFO:tensorflow:global_step/sec: 107.441
INFO:tensorflow:loss = 31.6266, step = 12301 (0.931 sec)
INFO:tensorflow:global_step/sec: 107.216
INFO:tensorflow:loss = 44.1324, step = 12401 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.443
INFO:tensorflow:loss = 34.6859, step = 12501 (0.931 sec)
INFO:tensorflow:global_step/sec: 107.266
INFO:tensorflow:loss = 29.4484, step = 12601 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.186
INFO:tensorflow:loss = 54.7993, step = 12701 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.139
INFO:tensorflow:loss = 25.4218, step = 12801 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.271
INFO:tensorflow:loss = 23.8161, step = 12901 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.277
INFO:tensorflow:loss = 24.5498, step = 13001 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.52
INFO:tensorflow:loss = 36.7875, step = 13101 (0.930 sec)
INFO:tensorflow:global_step/sec: 106.838
INFO:tensorflow:loss = 25.5743, step = 13201 (0.936 sec)
INFO:tensorflow:global_step/sec: 107.302
INFO:tensorflow:loss = 34.0919, step = 13301 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.445
INFO:tensorflow:loss = 33.1412, step = 13401 (0.931 sec)
INFO:tensorflow:global_step/sec: 106.735
INFO:tensorflow:loss = 28.3873, step = 13501 (0.937 sec)
INFO:tensorflow:global_step/sec: 107.221
INFO:tensorflow:loss = 18.9644, step = 13601 (0.933 sec)
INFO:tensorflow:global_step/sec: 108.184
INFO:tensorflow:loss = 43.8972, step = 13701 (0.924 sec)
INFO:tensorflow:global_step/sec: 107.439
INFO:tensorflow:loss = 10.6463, step = 13801 (0.931 sec)
INFO:tensorflow:global_step/sec: 107.398
INFO:tensorflow:loss = 15.1386, step = 13901 (0.931 sec)
INFO:tensorflow:global_step/sec: 106.823
INFO:tensorflow:loss = 21.4427, step = 14001 (0.936 sec)
INFO:tensorflow:global_step/sec: 106.828
INFO:tensorflow:loss = 9.19307, step = 14101 (0.936 sec)
INFO:tensorflow:global_step/sec: 107.288
INFO:tensorflow:loss = 24.6613, step = 14201 (0.932 sec)
INFO:tensorflow:global_step/sec: 106.763
INFO:tensorflow:loss = 6.44787, step = 14301 (0.937 sec)
INFO:tensorflow:global_step/sec: 107.369
INFO:tensorflow:loss = 26.7341, step = 14401 (0.931 sec)
INFO:tensorflow:global_step/sec: 107.08
INFO:tensorflow:loss = 17.9088, step = 14501 (0.934 sec)
INFO:tensorflow:global_step/sec: 106.596
INFO:tensorflow:loss = 6.79161, step = 14601 (0.938 sec)
INFO:tensorflow:global_step/sec: 106.063
INFO:tensorflow:loss = 16.1589, step = 14701 (0.943 sec)
INFO:tensorflow:global_step/sec: 107.352
INFO:tensorflow:loss = 9.55091, step = 14801 (0.932 sec)
INFO:tensorflow:global_step/sec: 106.747
INFO:tensorflow:loss = 10.9814, step = 14901 (0.937 sec)
INFO:tensorflow:global_step/sec: 107.847
INFO:tensorflow:loss = 7.66294, step = 15001 (0.927 sec)
INFO:tensorflow:global_step/sec: 104.862
INFO:tensorflow:loss = 4.43922, step = 15101 (0.954 sec)
INFO:tensorflow:global_step/sec: 108.932
INFO:tensorflow:loss = 15.6747, step = 15201 (0.918 sec)
INFO:tensorflow:global_step/sec: 107.909
INFO:tensorflow:loss = 10.7154, step = 15301 (0.927 sec)
INFO:tensorflow:global_step/sec: 103.498
INFO:tensorflow:loss = 13.2307, step = 15401 (0.966 sec)
INFO:tensorflow:global_step/sec: 107.489
INFO:tensorflow:loss = 8.0854, step = 15501 (0.930 sec)
INFO:tensorflow:global_step/sec: 109.331
INFO:tensorflow:loss = 12.1821, step = 15601 (0.915 sec)
INFO:tensorflow:global_step/sec: 107.584
INFO:tensorflow:loss = 12.294, step = 15701 (0.930 sec)
INFO:tensorflow:global_step/sec: 106.537
INFO:tensorflow:loss = 61.5253, step = 15801 (0.939 sec)
INFO:tensorflow:global_step/sec: 107.423
INFO:tensorflow:loss = 8.1819, step = 15901 (0.931 sec)
INFO:tensorflow:global_step/sec: 107.364
INFO:tensorflow:loss = 7.09414, step = 16001 (0.931 sec)
INFO:tensorflow:global_step/sec: 103.519
INFO:tensorflow:loss = 12.3909, step = 16101 (0.966 sec)
INFO:tensorflow:global_step/sec: 107.001
INFO:tensorflow:loss = 11.4822, step = 16201 (0.935 sec)
INFO:tensorflow:global_step/sec: 109.3
INFO:tensorflow:loss = 6.49143, step = 16301 (0.915 sec)
INFO:tensorflow:global_step/sec: 109.484
INFO:tensorflow:loss = 11.7731, step = 16401 (0.913 sec)
INFO:tensorflow:global_step/sec: 108.942
INFO:tensorflow:loss = 19.3688, step = 16501 (0.918 sec)
INFO:tensorflow:global_step/sec: 107.047
INFO:tensorflow:loss = 9.681, step = 16601 (0.934 sec)
INFO:tensorflow:global_step/sec: 105.395
INFO:tensorflow:loss = 12.1342, step = 16701 (0.949 sec)
INFO:tensorflow:global_step/sec: 104.754
INFO:tensorflow:loss = 5.07153, step = 16801 (0.954 sec)
INFO:tensorflow:global_step/sec: 107.31
INFO:tensorflow:loss = 42.5573, step = 16901 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.595
INFO:tensorflow:loss = 8.46675, step = 17001 (0.929 sec)
INFO:tensorflow:global_step/sec: 107.329
INFO:tensorflow:loss = 13.4557, step = 17101 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.598
INFO:tensorflow:loss = 6.96969, step = 17201 (0.929 sec)
INFO:tensorflow:global_step/sec: 107.648
INFO:tensorflow:loss = 11.6765, step = 17301 (0.929 sec)
INFO:tensorflow:global_step/sec: 107.26
INFO:tensorflow:loss = 12.9996, step = 17401 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.116
INFO:tensorflow:loss = 6.02258, step = 17501 (0.933 sec)
INFO:tensorflow:global_step/sec: 106.972
INFO:tensorflow:loss = 20.57, step = 17601 (0.935 sec)
INFO:tensorflow:global_step/sec: 107.526
INFO:tensorflow:loss = 5.76619, step = 17701 (0.930 sec)
INFO:tensorflow:global_step/sec: 107.057
INFO:tensorflow:loss = 14.0625, step = 17801 (0.934 sec)
INFO:tensorflow:global_step/sec: 108.311
INFO:tensorflow:loss = 498.567, step = 17901 (0.923 sec)
INFO:tensorflow:global_step/sec: 107.892
INFO:tensorflow:loss = 59.8736, step = 18001 (0.927 sec)
INFO:tensorflow:global_step/sec: 107.179
INFO:tensorflow:loss = 52.4814, step = 18101 (0.933 sec)
INFO:tensorflow:global_step/sec: 106.973
INFO:tensorflow:loss = 46.7329, step = 18201 (0.935 sec)
INFO:tensorflow:global_step/sec: 103.124
INFO:tensorflow:loss = 33.9443, step = 18301 (0.970 sec)
INFO:tensorflow:global_step/sec: 106.184
INFO:tensorflow:loss = 32.6561, step = 18401 (0.942 sec)
INFO:tensorflow:global_step/sec: 105.327
INFO:tensorflow:loss = 36.8528, step = 18501 (0.949 sec)
INFO:tensorflow:global_step/sec: 106.965
INFO:tensorflow:loss = 36.3321, step = 18601 (0.935 sec)
INFO:tensorflow:global_step/sec: 109.513
INFO:tensorflow:loss = 29.3537, step = 18701 (0.913 sec)
INFO:tensorflow:global_step/sec: 107.985
INFO:tensorflow:loss = 22.1533, step = 18801 (0.926 sec)
INFO:tensorflow:global_step/sec: 107.648
INFO:tensorflow:loss = 11.2524, step = 18901 (0.929 sec)
INFO:tensorflow:global_step/sec: 106.687
INFO:tensorflow:loss = 34.6025, step = 19001 (0.937 sec)
INFO:tensorflow:global_step/sec: 107.22
INFO:tensorflow:loss = 28.253, step = 19101 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.45
INFO:tensorflow:loss = 14.3627, step = 19201 (0.931 sec)
INFO:tensorflow:global_step/sec: 107.267
INFO:tensorflow:loss = 23.1622, step = 19301 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.179
INFO:tensorflow:loss = 13.557, step = 19401 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.118
INFO:tensorflow:loss = 23.2156, step = 19501 (0.934 sec)
INFO:tensorflow:global_step/sec: 107.329
INFO:tensorflow:loss = 27.2114, step = 19601 (0.932 sec)
INFO:tensorflow:global_step/sec: 101.271
INFO:tensorflow:loss = 14.0084, step = 19701 (0.987 sec)
INFO:tensorflow:global_step/sec: 102.407
INFO:tensorflow:loss = 20.3351, step = 19801 (0.977 sec)
INFO:tensorflow:global_step/sec: 104.565
INFO:tensorflow:loss = 17.0943, step = 19901 (0.956 sec)
INFO:tensorflow:global_step/sec: 106.964
INFO:tensorflow:loss = 30.746, step = 20001 (0.935 sec)
INFO:tensorflow:global_step/sec: 105.215
INFO:tensorflow:loss = 11.1444, step = 20101 (0.950 sec)
INFO:tensorflow:global_step/sec: 107.413
INFO:tensorflow:loss = 15.3931, step = 20201 (0.931 sec)
INFO:tensorflow:global_step/sec: 106.495
INFO:tensorflow:loss = 19.2516, step = 20301 (0.939 sec)
INFO:tensorflow:global_step/sec: 106.234
INFO:tensorflow:loss = 121.868, step = 20401 (0.941 sec)
INFO:tensorflow:global_step/sec: 107.371
INFO:tensorflow:loss = 40.0345, step = 20501 (0.931 sec)
INFO:tensorflow:global_step/sec: 106.998
INFO:tensorflow:loss = 16.1624, step = 20601 (0.935 sec)
INFO:tensorflow:global_step/sec: 107.176
INFO:tensorflow:loss = 53.3408, step = 20701 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.561
INFO:tensorflow:loss = 27.0281, step = 20801 (0.930 sec)
INFO:tensorflow:global_step/sec: 107.937
INFO:tensorflow:loss = 25.3671, step = 20901 (0.926 sec)
INFO:tensorflow:global_step/sec: 106.288
INFO:tensorflow:loss = 40.4708, step = 21001 (0.941 sec)
INFO:tensorflow:global_step/sec: 107.416
INFO:tensorflow:loss = 24.8571, step = 21101 (0.931 sec)
INFO:tensorflow:global_step/sec: 107.399
INFO:tensorflow:loss = 29.6436, step = 21201 (0.931 sec)
INFO:tensorflow:global_step/sec: 106.68
INFO:tensorflow:loss = 17.2048, step = 21301 (0.937 sec)
INFO:tensorflow:global_step/sec: 107.835
INFO:tensorflow:loss = 29.9399, step = 21401 (0.927 sec)
INFO:tensorflow:global_step/sec: 105.215
INFO:tensorflow:loss = 12.9063, step = 21501 (0.950 sec)
INFO:tensorflow:global_step/sec: 107.468
INFO:tensorflow:loss = 15.8529, step = 21601 (0.931 sec)
INFO:tensorflow:global_step/sec: 105.956
INFO:tensorflow:loss = 11.2438, step = 21701 (0.944 sec)
INFO:tensorflow:global_step/sec: 107.145
INFO:tensorflow:loss = 40.0604, step = 21801 (0.933 sec)
INFO:tensorflow:global_step/sec: 108.602
INFO:tensorflow:loss = 73.5826, step = 21901 (0.921 sec)
INFO:tensorflow:global_step/sec: 108.608
INFO:tensorflow:loss = 29.2865, step = 22001 (0.921 sec)
INFO:tensorflow:global_step/sec: 107.157
INFO:tensorflow:loss = 69.5928, step = 22101 (0.933 sec)
INFO:tensorflow:global_step/sec: 108.051
INFO:tensorflow:loss = 90.4058, step = 22201 (0.925 sec)
INFO:tensorflow:global_step/sec: 107.277
INFO:tensorflow:loss = 9.10217, step = 22301 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.009
INFO:tensorflow:loss = 21.8856, step = 22401 (0.934 sec)
INFO:tensorflow:global_step/sec: 107.905
INFO:tensorflow:loss = 20.5334, step = 22501 (0.927 sec)
INFO:tensorflow:global_step/sec: 107.089
INFO:tensorflow:loss = 13.6869, step = 22601 (0.934 sec)
INFO:tensorflow:global_step/sec: 107.104
INFO:tensorflow:loss = 13.1339, step = 22701 (0.934 sec)
INFO:tensorflow:global_step/sec: 108.242
INFO:tensorflow:loss = 48.243, step = 22801 (0.924 sec)
INFO:tensorflow:global_step/sec: 105.801
INFO:tensorflow:loss = 37.6556, step = 22901 (0.945 sec)
INFO:tensorflow:global_step/sec: 107.029
INFO:tensorflow:loss = 25.5693, step = 23001 (0.934 sec)
INFO:tensorflow:global_step/sec: 107.251
INFO:tensorflow:loss = 18.8527, step = 23101 (0.932 sec)
INFO:tensorflow:global_step/sec: 107.224
INFO:tensorflow:loss = 15.3408, step = 23201 (0.933 sec)
INFO:tensorflow:global_step/sec: 108.272
INFO:tensorflow:loss = 9.74794, step = 23301 (0.924 sec)
INFO:tensorflow:global_step/sec: 106.192
INFO:tensorflow:loss = 7.08407, step = 23401 (0.942 sec)
INFO:tensorflow:global_step/sec: 107.732
INFO:tensorflow:loss = 310.551, step = 23501 (0.928 sec)
INFO:tensorflow:global_step/sec: 106.775
INFO:tensorflow:loss = 82.5201, step = 23601 (0.937 sec)
INFO:tensorflow:global_step/sec: 106.662
INFO:tensorflow:loss = 33.618, step = 23701 (0.937 sec)
INFO:tensorflow:global_step/sec: 107.135
INFO:tensorflow:loss = 98.7111, step = 23801 (0.933 sec)
INFO:tensorflow:global_step/sec: 107.064
INFO:tensorflow:loss = 46.9719, step = 23901 (0.934 sec)
INFO:tensorflow:global_step/sec: 100.65
INFO:tensorflow:loss = 32.1506, step = 24001 (0.994 sec)
INFO:tensorflow:global_step/sec: 108.046
INFO:tensorflow:loss = 30.4943, step = 24101 (0.926 sec)
INFO:tensorflow:global_step/sec: 106.233
INFO:tensorflow:loss = 75.8596, step = 24201 (0.941 sec)
INFO:tensorflow:global_step/sec: 107.909
INFO:tensorflow:loss = 16.4762, step = 24301 (0.927 sec)
INFO:tensorflow:global_step/sec: 107.435
INFO:tensorflow:loss = 15.648, step = 24401 (0.931 sec)
INFO:tensorflow:global_step/sec: 107.753
INFO:tensorflow:loss = 27.7351, step = 24501 (0.928 sec)
INFO:tensorflow:global_step/sec: 108.195
INFO:tensorflow:loss = 19.3522, step = 24601 (0.924 sec)
INFO:tensorflow:global_step/sec: 106.403
INFO:tensorflow:loss = 15.5935, step = 24701 (0.940 sec)
INFO:tensorflow:global_step/sec: 107.07
INFO:tensorflow:loss = 15.0167, step = 24801 (0.933 sec)
INFO:tensorflow:global_step/sec: 109.552
INFO:tensorflow:loss = 10.7183, step = 24901 (0.913 sec)
INFO:tensorflow:Saving checkpoints for 25000 into trained_models/seq-model-02/model.ckpt.
INFO:tensorflow:Loss for final step: 12.8604.

* data input_fn:
================
Input file(s): data/seq02.train.csv
Batch size: 200
Epoch Count: 1
Mode: eval
Shuffle: False
================

INFO:tensorflow:Starting evaluation at 2017-11-28-18:49:08
INFO:tensorflow:Restoring parameters from trained_models/seq-model-02/model.ckpt-25000
INFO:tensorflow:Finished evaluation at 2017-11-28-18:49:10
INFO:tensorflow:Saving dict for global step 25000: global_step = 25000, loss = 16.9192, mae = 2.93555, rmse = 4.11225
INFO:tensorflow:Restoring parameters from trained_models/seq-model-02/model.ckpt-25000
INFO:tensorflow:Assets added to graph.
INFO:tensorflow:No assets to write.
INFO:tensorflow:SavedModel written to: b"trained_models/seq-model-02/export/forecast/temp-b'1511894954'/saved_model.pbtxt"

* data input_fn:
================
Input file(s): data/seq02.train.csv
Batch size: 200
Epoch Count: 5000
Mode: train
Shuffle: True
================

INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Restoring parameters from trained_models/seq-model-02/model.ckpt-25000
INFO:tensorflow:Saving checkpoints for 25001 into trained_models/seq-model-02/model.ckpt.
INFO:tensorflow:loss = 15.1169, step = 25001
INFO:tensorflow:global_step/sec: 61.0345
INFO:tensorflow:loss = 19.7777, step = 25101 (1.640 sec)
INFO:tensorflow:global_step/sec: 100.684
INFO:tensorflow:loss = 16.2851, step = 25201 (0.993 sec)
INFO:tensorflow:global_step/sec: 103.516
INFO:tensorflow:loss = 9.39814, step = 25301 (0.966 sec)
INFO:tensorflow:global_step/sec: 103.303
INFO:tensorflow:loss = 19.437, step = 25401 (0.968 sec)
INFO:tensorflow:global_step/sec: 102.53
INFO:tensorflow:loss = 8.39387, step = 25501 (0.975 sec)
INFO:tensorflow:global_step/sec: 102.931
INFO:tensorflow:loss = 88.8126, step = 25601 (0.971 sec)
INFO:tensorflow:global_step/sec: 103.128
INFO:tensorflow:loss = 19.8987, step = 25701 (0.970 sec)
INFO:tensorflow:global_step/sec: 103.766
INFO:tensorflow:loss = 45.2823, step = 25801 (0.964 sec)
INFO:tensorflow:global_step/sec: 103.27
INFO:tensorflow:loss = 9.37735, step = 25901 (0.968 sec)
INFO:tensorflow:global_step/sec: 97.9022
INFO:tensorflow:loss = 34.4669, step = 26001 (1.021 sec)
INFO:tensorflow:global_step/sec: 91.1345
INFO:tensorflow:loss = 13.694, step = 26101 (1.097 sec)
INFO:tensorflow:global_step/sec: 101.738
INFO:tensorflow:loss = 11.3014, step = 26201 (0.983 sec)
INFO:tensorflow:global_step/sec: 103.431
INFO:tensorflow:loss = 25.1076, step = 26301 (0.967 sec)
INFO:tensorflow:global_step/sec: 103.44
INFO:tensorflow:loss = 15.2882, step = 26401 (0.967 sec)
INFO:tensorflow:global_step/sec: 103.687
INFO:tensorflow:loss = 15.075, step = 26501 (0.965 sec)
INFO:tensorflow:global_step/sec: 102.896
INFO:tensorflow:loss = 6.34362, step = 26601 (0.972 sec)
INFO:tensorflow:global_step/sec: 102.571
INFO:tensorflow:loss = 8.00465, step = 26701 (0.975 sec)
INFO:tensorflow:global_step/sec: 103.142
INFO:tensorflow:loss = 18.5518, step = 26801 (0.970 sec)
INFO:tensorflow:global_step/sec: 103.434
INFO:tensorflow:loss = 8.25713, step = 26901 (0.967 sec)
INFO:tensorflow:global_step/sec: 103.025
INFO:tensorflow:loss = 36.2909, step = 27001 (0.971 sec)
INFO:tensorflow:global_step/sec: 103.584
INFO:tensorflow:loss = 16.0341, step = 27101 (0.965 sec)
INFO:tensorflow:global_step/sec: 103.126
INFO:tensorflow:loss = 133.178, step = 27201 (0.970 sec)
INFO:tensorflow:global_step/sec: 103.293
INFO:tensorflow:loss = 54.2295, step = 27301 (0.968 sec)
INFO:tensorflow:global_step/sec: 103.056
INFO:tensorflow:loss = 35.7114, step = 27401 (0.970 sec)
INFO:tensorflow:global_step/sec: 103.314
INFO:tensorflow:loss = 35.6843, step = 27501 (0.968 sec)
INFO:tensorflow:global_step/sec: 102.739
INFO:tensorflow:loss = 23.2631, step = 27601 (0.973 sec)
INFO:tensorflow:global_step/sec: 102.55
INFO:tensorflow:loss = 25.6496, step = 27701 (0.975 sec)
INFO:tensorflow:global_step/sec: 103.455
INFO:tensorflow:loss = 90.7616, step = 27801 (0.967 sec)
INFO:tensorflow:global_step/sec: 102.797
INFO:tensorflow:loss = 24.2504, step = 27901 (0.973 sec)
INFO:tensorflow:global_step/sec: 103.243
INFO:tensorflow:loss = 26.095, step = 28001 (0.969 sec)
INFO:tensorflow:global_step/sec: 103.496
INFO:tensorflow:loss = 37.4449, step = 28101 (0.966 sec)
INFO:tensorflow:global_step/sec: 99.7283
INFO:tensorflow:loss = 16.8191, step = 28201 (1.003 sec)
INFO:tensorflow:global_step/sec: 102.996
INFO:tensorflow:loss = 22.3019, step = 28301 (0.971 sec)
INFO:tensorflow:global_step/sec: 100.857
INFO:tensorflow:loss = 33.6963, step = 28401 (0.991 sec)
INFO:tensorflow:global_step/sec: 102.533
INFO:tensorflow:loss = 103.184, step = 28501 (0.975 sec)
INFO:tensorflow:global_step/sec: 102.87
INFO:tensorflow:loss = 17.3864, step = 28601 (0.972 sec)
INFO:tensorflow:global_step/sec: 103.25
INFO:tensorflow:loss = 14.7848, step = 28701 (0.968 sec)
INFO:tensorflow:global_step/sec: 102.438
INFO:tensorflow:loss = 14.7516, step = 28801 (0.976 sec)
INFO:tensorflow:global_step/sec: 103.301
INFO:tensorflow:loss = 15.7594, step = 28901 (0.968 sec)
INFO:tensorflow:global_step/sec: 103.588
INFO:tensorflow:loss = 15.0374, step = 29001 (0.965 sec)
INFO:tensorflow:global_step/sec: 102.797
INFO:tensorflow:loss = 9.90007, step = 29101 (0.973 sec)
INFO:tensorflow:global_step/sec: 103.901
INFO:tensorflow:loss = 31.6541, step = 29201 (0.963 sec)
INFO:tensorflow:global_step/sec: 103.352
INFO:tensorflow:loss = 10.4815, step = 29301 (0.968 sec)
INFO:tensorflow:global_step/sec: 103.221
INFO:tensorflow:loss = 9.89371, step = 29401 (0.969 sec)
INFO:tensorflow:global_step/sec: 102.989
INFO:tensorflow:loss = 14.3944, step = 29501 (0.971 sec)
INFO:tensorflow:global_step/sec: 83.3394
INFO:tensorflow:loss = 8.78345, step = 29601 (1.200 sec)
INFO:tensorflow:global_step/sec: 103.208
INFO:tensorflow:loss = 24.7691, step = 29701 (0.969 sec)
INFO:tensorflow:global_step/sec: 101.845
INFO:tensorflow:loss = 15.2601, step = 29801 (0.982 sec)
INFO:tensorflow:global_step/sec: 103.176
INFO:tensorflow:loss = 9.99201, step = 29901 (0.969 sec)
INFO:tensorflow:Saving checkpoints for 30000 into trained_models/seq-model-02/model.ckpt.
INFO:tensorflow:Loss for final step: 17.4547.

* data input_fn:
================
Input file(s): data/seq02.train.csv
Batch size: 200
Epoch Count: 1
Mode: eval
Shuffle: False
================

INFO:tensorflow:Starting evaluation at 2017-11-28-18:50:23
INFO:tensorflow:Restoring parameters from trained_models/seq-model-02/model.ckpt-30000
INFO:tensorflow:Finished evaluation at 2017-11-28-18:50:25
INFO:tensorflow:Saving dict for global step 30000: global_step = 30000, loss = 13.2095, mae = 2.52851, rmse = 3.63426
INFO:tensorflow:Restoring parameters from trained_models/seq-model-02/model.ckpt-30000
INFO:tensorflow:Assets added to graph.
INFO:tensorflow:No assets to write.
INFO:tensorflow:SavedModel written to: b"trained_models/seq-model-02/export/forecast/temp-b'1511895026'/saved_model.pbtxt"
.......................................
Experiment finished at 18:50:27

Experiment elapsed time: 339.007524 seconds

















In [16]:



    


train_input_fn = lambda: csv_input_fn(files_name_pattern= TRAIN_DATA_FILE, 
                                      mode= tf.estimator.ModeKeys.EVAL,
                                      batch_size= TRAIN_DATA_SIZE)

test_input_fn = lambda: csv_input_fn(files_name_pattern= TEST_DATA_FILE, 
                                      mode= tf.estimator.ModeKeys.EVAL,
                                      batch_size= TEST_DATA_SIZE)

estimator = create_estimator(run_config, hparams)

train_results = estimator.evaluate(input_fn=train_input_fn, steps=1)
train_rmse = str(train_results["rmse"])
print()
print("######################################################################################")
print("# Train RMSE: {} - {}".format(train_rmse, train_results))
print("######################################################################################")

test_results = estimator.evaluate(input_fn=test_input_fn, steps=1)
test_rmse = str(test_results["rmse"])
print()
print("######################################################################################")
print("# Test RMSE: {} - {}".format(test_rmse, test_results))
print("######################################################################################")



    


















    






INFO:tensorflow:Using config: {'_model_dir': 'trained_models/seq-model-02', '_tf_random_seed': 19830610, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x126aa35c0>, '_task_type': 'worker', '_task_id': 0, '_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}

Estimator Type: <class 'tensorflow.python.estimator.estimator.Estimator'>


* data input_fn:
================
Input file(s): data/seq02.train.csv
Batch size: 1200
Epoch Count: 1
Mode: eval
Shuffle: False
================

INFO:tensorflow:Starting evaluation at 2017-11-28-18:50:30
INFO:tensorflow:Restoring parameters from trained_models/seq-model-02/model.ckpt-30000
INFO:tensorflow:Evaluation [1/1]
INFO:tensorflow:Finished evaluation at 2017-11-28-18:50:32
INFO:tensorflow:Saving dict for global step 30000: global_step = 30000, loss = 13.2079, mae = 2.52851, rmse = 3.63426

######################################################################################
# Train RMSE: 3.63426 - {'loss': 13.207876, 'mae': 2.5285063, 'rmse': 3.6342642, 'global_step': 30000}
######################################################################################

* data input_fn:
================
Input file(s): data/seq02.test.csv
Batch size: 300
Epoch Count: 1
Mode: eval
Shuffle: False
================

INFO:tensorflow:Starting evaluation at 2017-11-28-18:50:35
INFO:tensorflow:Restoring parameters from trained_models/seq-model-02/model.ckpt-30000
INFO:tensorflow:Evaluation [1/1]
INFO:tensorflow:Finished evaluation at 2017-11-28-18:50:37
INFO:tensorflow:Saving dict for global step 30000: global_step = 30000, loss = 21.1006, mae = 3.04524, rmse = 4.59354

######################################################################################
# Test RMSE: 4.59354 - {'loss': 21.100595, 'mae': 3.0452378, 'rmse': 4.5935383, 'global_step': 30000}
######################################################################################

6. Prediction





In [17]:



    


import itertools

LEN = 999

predict_input_fn = lambda: csv_input_fn(files_name_pattern= TRAIN_DATA_FILE, 
                                      mode= tf.estimator.ModeKeys.PREDICT,
                                      batch_size=TRAIN_DATA_SIZE)

predictions = list(itertools.islice(estimator.predict(input_fn=predict_input_fn),LEN))
prediction_outputs = list(map(lambda item: list(item["values"]),predictions[:3]))
print()
print("Predicted Values: {}".format(prediction_outputs))



    


















    






* data input_fn:
================
Input file(s): data/seq02.train.csv
Batch size: 1200
Epoch Count: 1
Mode: infer
Shuffle: False
================

WARNING:tensorflow:Input graph does not contain a QueueRunner. That means predict yields forever. This is probably a mistake.
INFO:tensorflow:Restoring parameters from trained_models/seq-model-02/model.ckpt-30000

Predicted Values: [[-19.600773, -18.351267, -19.886063, -18.639763], [180.13437, 181.39165, 177.92723, 178.25304], [-368.42282, 366.8053, -370.4343, 369.09964]]

















In [18]:



    


forecasted_values = list(map(lambda item: list(item["values"])[0],predictions))
forecasted_seq1 = []
forecasted_seq2 = []
forecasted_seq3 = []

i = 0

while i < LEN:
    forecasted_seq1 += [forecasted_values[i]]
    forecasted_seq2 += [forecasted_values[i+1]]
    forecasted_seq3 += [forecasted_values[i+2]]
    i += 3



    











In [19]:



    


l = 100
x = np.array(range(0, l ))

plt.close('all')
plt.figure(figsize=(20, 5))

plt.subplot(1,3,1)
plt.title("Forecasted Pattern 1")
y = forecasted_seq1[:l ]
plt.plot(x,y)

plt.subplot(1,3,2)
plt.title("Forecasted Pattern 2")
y = forecasted_seq2[:l ]
plt.plot(x/3,y)

plt.subplot(1,3,3)
plt.title("Forecasted Pattern 3")
y = forecasted_seq3[: l]
plt.plot(x,y)
    
plt.xlabel("Element Index")
plt.ylabel("Value")
plt.show()

7. Serving the Exported Model





In [20]:



    


import os

export_dir = model_dir +"/export/forecast"

saved_model_dir = export_dir + "/" + os.listdir(path=export_dir)[-1] 

print(saved_model_dir)

predictor_fn = tf.contrib.predictor.from_saved_model(
    export_dir = saved_model_dir,
    signature_def_key="predictions"
)

output = predictor_fn({'csv_rows': ['1,2,3,4,5,6,7,8,9,10,9,8,7,6,5,4']})
print(output)



    


















    






trained_models/seq-model-02/export/forecast/1511895026
INFO:tensorflow:Restoring parameters from b'trained_models/seq-model-02/export/forecast/1511895026/variables/variables'
{'values': array([[ 4.62223148,  4.57558107,  4.18876743,  4.63509703]], dtype=float32)}

















In [ ]:

Content source: GoogleCloudPlatform/tf-estimator-tutorials

Similar notebooks:

notebook.community | gallery | about